discard

set more off 

/** this do-file renames and recodes the self-reported health variables 
	so that they are consistent across waves **/ 	
			
		use "$healthsave\merged health data cleaned_w7.dta"	, clear 
		
	
	*******************************************************
	** we now derive the following set of variables: 	 ** 
	** 		- yes response to ever diagnosed 			 ** 
	** 		- past report fed forward 					 ** 
	** 		- attitude to previous report 				 ** 
	** 				(confirmation or dispute + reason)   ** 
	******************************************************* 
	
		lab define ever 		   -1 "Missing" 				///
									0 "No diagnosis reported" 	///
									1 "Diagnosis reported" 
	
		lab define ff	    		0 "Not fed forward" 	 ///
									1 "Fed forward" 

		 lab define disp 			-1 "Missing" ///
									1 "Not fed-forward" ///
									2 "Confirm previous report" 	///
									3 "Dispute - never had it" 		///
									4 "Dispute - not prev but now" 	///
									5 "Dispute - misdiagnosed" 		 		
									
		lab define aged 			0 "Not applicable" -1 "Missing" 
			
		lab define yrd 				0 "Not applicable" -1 "Missing" 				


		tokenize LungDisease Asthma Arthritis Osteoporosis Cancer Parkinsons ///
				 PsychiatricProblems Alzheimers Dementia BloodDisorder 		 ///
				 Hypertension Angina HeartAttack HeartFailure HeartMurmur	 ///
				 HeartRhythm Diabetes Stroke Cholesterol	
		
		foreach x in lu as ar os ca pd ps ad de bl bp an mi hf hm hr di st ch { 
			forv i = 0/7 { 
				
				gen ever`x'_w`i' = . 
				lab var ever`x'_w`i' "Reports diagnosis of `1'" 
				lab val ever`x'_w`i' ever 
				
				gen ff`x'_w`i' = . 
				lab var ff`x'_w`i' "`1' fed-forward(?)" 
				lab val ff`x'_w`i' ff 
				
				gen disp`x'_w`i' = .
				lab var disp`x'_w`i' "Dispute/confirmation information" 
				lab val disp`x'_w`i' disp 
				
				gen aged`x'_w`i' = .
				lab var aged`x'_w`i'  "Age diagnosed with `1'" 
				lab val aged`x'_w`i' aged 

				gen yrd`x'_w`i' = .
				lab var yrd`x'_w`i'  "Year diagnosed with `1'" 
				lab val yrd`x'_w`i' yrd 
				
				gen mnthd`x'_w`i' = .
				lab var mnthd`x'_w`i'  "Month diagnosed with `1'" 
				lab val mnthd`x'_w`i' mnthd 					
			}
		mac shift 
		}
		

		
		
		*------------------------------*
		* DERIVE AGE/YEAR OF DIAGNOSIS *
		*------------------------------*
		
		* rename variables
		
			forv i=1/7 {
				ren heprkw`i' heagpw`i' 
			}
			
			forv i=2/7 {
				ren heprkrw`i' heagprw`i' 
				ren heprkryw`i' heagpryw`i' 
			}
			forv i = 1/2 {
				gen heagkw`i' = . 
				gen heagkryw`i'  = . 
				gen heagkrw`i'  = . 

			}
			
			foreach x in a b c d e f g p h i j  { 
				gen heag`x'ryw1 = .
				gen heag`x'rw1  = .
			} 
			
			
			** two cases of diabetes when diagnosis age is 1 year higher than current age. 
			** otherwise all seems consistent. 
				list idauniq if  heagdw1>agew1 & heagdw1!=.
				replace heagdw1 = agew1 if heagdw1>agew1 & heagdw1!=.

			foreach x in a b c d e f g p h i j k { 
				forv i = 1/7 { 
		*			di in red "`x'"
					assert heag`x'w`i'   <= agew`i'  |  heag`x'w`i' ==. 
					assert heag`x'ryw`i' <= intyear_w`i' | heag`x'ryw`i'==.
				}
				mac shift 
				}

			tokenize a b c d e f g p h i j k 
			foreach x in an mi hf di st ar ca pd ps ad de bl {
			
				forv i = 1/7 { 
				replace aged`x'_w`i' = heag`1'w`i' 
				recode aged`x'_w`i' (-1 = 0)  (-9/-8 = -1) 
				
				replace yrd`x'_w`i'  = heag`1'ryw`i' 
				recode yrd`x'_w`i' (-1 = 0)  (-9/-8 = -1) 
				
				replace mnthd`x'_w`i' = heag`1'rw`i' 
				recode mnthd`x'_w`i' (-1 = 0)  (-9/-8 = -1) 
			}
			mac shift 
			}		
		
			
		*----------* 
		*- WAVE 0 -*
		*----------* 		
		
			* we have answers to 'which long-standing illness limits you?' 
			* and also, for some conditions, whether an individual has ever
			* been diagnosed by a doctor 

		/* Those answering 'no' to whether they have a longstanding illness 
			have 'item not applicable' recorded in the illsm* variable. */
			assert illsm1 == -1 if longillw0 == 2 

		
		
		/** have commented this out because of a decision 30.6.14 only 
			to use doctor diagnoses, and NOT self-reports **/ 		
		
/*
		tokenize 1 4 22 23 34 38
				
		foreach x in ca ps lu as ar bl {

			replace ever`x'_w0 = -1 if longillw0==-1 	/* missings */		
			replace	ever`x'_w0 =  0 if longillw0==1 | longillw0==2 
			replace ever`x'_w0 = -1 if illsm1<=-7 
		
			forv i = 1/6 {
				replace ever`x'_w0 = 1 if illsm`i'w0==`1'
			}
		
		mac shift 
		}			*/

						
		/* coding up answers to: 
			have you ever been diagnosed by a doctor with condition X? */
				
			* override long-standing illness question 
			* information only available for those in HSE '98 and '99 
			* (or refresher sample)
			
			** for heart murmurs, the w3&4 refresher samples only 
			** have the 'what age were you diagnosed' variable. 
				replace murmur1w0 = 1 if (ageinmurw0 == -8 | (ageinmurw0>=0 & ageinmurw0!=.)) & murmur1w0==. 
				replace murmur1w0 = 2 if (ageinmurw0 == -1 | ageinmurw0==-9) & murmur1w0==. 

			tokenize angidef strodefw0 bp1w0 heartdef murmur iregdefw0 diabete2
			foreach x in an st bp mi hm hr di { 
				recode `1' (-1 = .) (-9/-8 = -1) (2=0) 
				replace ever`x'_w0 = `1'
				mac shift 
			}
		
					
		/** for documentation **/
		
			cd "P:\ELSA\Healthdynamics\documentation\tables\" 
			preserve
			tokenize     2 15 17 
			foreach x in di st bp { 
				gen longill`x' = 0 if longillw0 == 1 | longillw0==2 
				forv i = 1/6 { 
					replace longill`x' = 1 if illsm`i'w0 == `1' 
				}
			mac shift 
			}
			
			foreach x in di st bp { 
				gen diag`x' = 0 if ever`x'_w0 == 0 
				replace diag`x' = 1 if ever`x'_w0 ==1 
			}

			foreach x in di st bp { 
				gen false_pos_`x' = longill`x' == 1 & diag`x' == 0 if longill`x'!=. & diag`x'!=. 
				gen false_neg_`x' = longill`x' == 0 & diag`x' == 1 if longill`x'!=. & diag`x'!=. 
				replace diag`x' = . if longill`x'==. 
			} 
			
			keep idauniq diag* false_pos* false_neg* 
			reshape long false_pos_ false_neg_ diag , i(idauniq) j(condition) s 
			tabout condition using HSE_longill_error_rates.txt ///
					, c(mean diag mean false_pos_ mean false_neg_ count diag) ///
					f(5) sum replace 
			
			restore 
			
	*----------* 
	*- WAVE 1 -*
	*----------* 
	
		
	********************	
	** ever diagnosed **
 	********************
		
	tokenize 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8
		
	foreach x in lu as ar os ca pd ps ad de bp an mi hf hm hr di st {
		if ("`x'"=="lu" | "`x'"=="as" | "`x'"=="ar" | "`x'"=="os" | "`x'"=="ca" ///
		    | "`x'"=="pd" | "`x'"=="ps" | "`x'"=="ad" | "`x'"=="de") {
			
			replace ever`x'_w1 = -1 if hedib01w1<0 						/* missings - refuse/don't know */
			replace ever`x'_w1 = 0  if hedib01w1>=0 & hedib01w1!=. 		/* successful response */
			replace ever`x'_w1 = 1  if hedib01w1==`1' 					/* reports condition */
						
			forv i =2/7 {
				assert hedib0`i'w1!=. if hedib01w1!=. 					
				assert hedib0`i'w1==-1 if hedib01w1<0 					
				replace ever`x'_w1 = 1 if hedib0`i'w1==`1' 				/* reports condition */
			}
		}
					
		else {
			replace ever`x'_w1 = -1 if hedim01w1<0 
			replace ever`x'_w1 = 0  if hedim01w1>=0 & hedim01w1!=. 
			replace ever`x'_w1 = 1  if hedim01w1==`1' 
						
			forv i =2/7 {
				assert hedim0`i'w1!=. if hedim01w1!=. 
				assert hedim0`i'w1==-1 if hedim01w1<0 									
				replace ever`x'_w1 = 1 if hedim0`i'w1==`1' 
			}
		}	
		mac shift
	}
	
	
	
	** check that reports of heart attacks are fed into prevalence vars **
			
		assert evermi_w1 == 1 if heyrbw1==1 

		
		
		
		
	*----------* 
	*- WAVE 2 -*
	*----------* 	
		

		** 					chronic / CVD 
		** feed forward: 	hedibw  / hediaw 
		** confirmation: 	hediad  / hediac 
		** dispute reason:  hediam  / hedian 
		** new report: 	    hedib0? / hedim0?
		
		** note that hediac9, hedaw9 and hedian9 
		** do not refer to cholesterol 
		** WHEREAS CODE 9 IN HEDIM0? DOES REFER TO CHOLESTEROL. 		
		
		replace hediac9w2 = . 
		replace hediaw9w2 = .
		replace hedian9w2 = .
		
		** diagnosis age: 	aged_w? 
		** diagnosis time: yrd_w? , mnthd_w? / 
		
		** make sure variable relationships make sense:
		** 		(i)   feed forward necessary and sufficient for confirmation 
		**		(ii)  reason only given for dispute if disputed 
		** 		(ii)  change dispute reason to missing if confirmation refused. 
		** 		(iv)  new report necesary and sufficient(?) for age/time of diagnosis 
		** 		(v)   age/time of diagnosis missing if refusal, 
		**				n/a if didn't report new diagnosis 	
		
		** ff is ncessary and sufficient for confirmation request
		
		forv i = 1/9 { 			
			assert hediad`i' != -1 if hedibw`i'w2 ==`i' 
			assert hediac`i' != -1 if hediaw`i'w2 ==`i' 
		}
		

		** sometimes a reason for a dispute is given, despite an apparent 
		** confirmation. Likely to be because routing didn't allow interviewer 
		** to recode reason for diapute after it turned out that the respondent 
		** actually confirmed the previous report. 
		
		forv i = 1/9 { 
			di in red "`i'" 
			list idauniq if hediam`i'w2!=-1 & hediam`i'w2!=2 & hediad`i'w2 == 1 // confirms that has condition but has dispute response that is neither N/A nor no longer has
			replace hediam`i'w2 = -1 if hediad`i'w2 == 1 // recode dispute to N/A if have confirmation
			
			list idauniq if hedian`i'w2!=-1 & hediac`i'w2 == 1 & hedian`i'w2!=2 			
			replace hedian`i'w2 = -1 if hediac`i'w2 == 1 
		}
		
				
		** confirmations should always pass properly on to reasons for disputes. 
		** in some cases it looks like the confirmation question hasn't been 
		** asked, despite there being a subsequent dispute response. 
		
		** assume that the later reason for dispute corresponds to a real 
		** confirmation. 
		
		forv i = 1/9 { 
		di in red "`i'" 
			list idauniq if hediad`i'!=2 & hediam`i'w2 >=1 & hediam`i'w2<=3 // confirms but then gives a reason for dispute 
			replace hediad`i'w2 = 2 if hediam`i'w2 >=1 & hediam`i'w2<=3 // recode confirmation to dispute if gives reason for dispute
			
			list idauniq if hediac`i'!=2 & hedian`i'w2 >=1 & hedian`i'w2<=3 			
			replace hediac`i'w2 = 2 if hedian`i'w2 >=1 & hedian`i'w2<=3 
		} 

		** otherwise, assume that if the confirmation question wasn't asked 
		** effectively the condition wasn't fed forward properly. 
				
		forv i = 1/9 { 
			
			** replace ff equal to zero 
			replace hedibw`i'w2 = -1 if hediad`i'w2==-4 | hediad`i'w2==3 // -4 is asked because of feed forward problem and 3 is not read out as didn't make sense
			replace hediaw`i'w2 = -1 if hediac`i'w2==-4 | hediac`i'w2==-3 | hediac`i'w2==3 
 
			** replace confirmations equal to n/a
			recode hediad`i'w2 (-4 = -1) (3 = -1) 
			recode hediac`i'w2 (-4/-3 = -1) (3 = -1) 
		}	
		
		** make sure that dispute reasons are only given 
		** if there is NO confirmation 
		
		forv i = 1/9 { 
			assert hediam`i'w2 !=-1 if hediad`i'w2 ==2 
			assert hediad`i'w2 !=2  if hediam`i'w2==-1 
			
			assert hedian`i'w2 !=-1 if hediac`i'w2 ==2 
			assert hediac`i'w2 !=2  if hedian`i'w2==-1 				
		} 

		** change dispute reason to missing if confirmation refused. 
			forv i = 1/9 { 
			assert hediam`i'w2 == -1 if  hediad`i'w2 <-1 
			assert hedian`i'w2 == -1 if  hediac`i'w2 <-1 
				replace hediam`i'w2 = hediad`i'w2 if  hediad`i'w2 <-1 
				replace hedian`i'w2 = hediac`i'w2 if  hediac`i'w2 <-1 
			}
	
		
		** if says 'no longer have it' as dispute reason, take this as 
		** confirmation of previous dispute. 
		
			forv i = 1/9 {
				replace hediad`i'w2 = 1 if hediam`i'w2 == 2 
				replace hediac`i'w2 = 1 if hedian`i'w2 == 2 
			
				replace hediam`i'w2 = -1 if hediam`i'w2 == 2 
				replace hediac`i'w2 = -1 if hedian`i'w2 == 2
			}
			
		 
		 
		tokenize 1 2 3 4 5 6 7 8 9 1 2 3 4 5 6 7 8 9
		foreach x in lu as ar os ca pd ps ad de bp an mi hf hm hr di st ch {

		if "`x'"=="lu" | "`x'"=="as" | "`x'"=="ar" | "`x'"=="os" | "`x'"=="ca" ///
		    | "`x'"=="pd" | "`x'"=="ps" | "`x'"=="ad" | "`x'"=="de"  {
				
		** 					chronic / CVD 
		** feed forward: 	hedibw  / hediaw 
		** confirmation: 	hediad  / hediac 
		** dispute reason:  hediam  / hedian 
		** new report: 	    hedib0? / hedim0?
				
		** ff information 
		******************
		
			replace ff`x'_w2 = 0 if inwave2==1
			replace ff`x'_w2 = 0 if hedibw`1'w2==-1 
			replace ff`x'_w2 = 1 if hedibw`1'w2==`1' 
					
		
		
		** dispute information 
		***********************

			replace disp`x'_w2 = -1  if hediad`1'w2<=-3 | hediam`1'w2<=-3 		/* missing responses */
			replace disp`x'_w2 =  1  if ff`x'_w2 == 0 		/* nothing ff */
			replace disp`x'_w2 =  2  if hediad`1'w2==1 		/* confirm */
			replace disp`x'_w2 =  3  if hediam`1'w2==1		/* never had it */
			replace disp`x'_w2 =  4  if hediam`1'w2==3		/* not prev but now */
			assert disp`x'_w2 !=. if inwave2==1 			
		
		** ever diagnosed 
		******************
			
			* confirmation of previous diagnosis
			replace ever`x'_w2 = -1 if disp`x'_w2 == -1 	/* missing confimation info */
			replace ever`x'_w2 =  0 if disp`x'_w2 == 3 		/* never had it */
			replace ever`x'_w2 =  1 if disp`x'_w2 == 2 | disp`x'_w2 == 4 /* have now */
			
			* fresh reports 
			replace ever`x'_w2 = -1 if ever`x'_w2 == . & hedib01w2 <=-8 
			replace ever`x'_w2 =  0 if ever`x'_w2 == . & hedib01w2 == 96 
			
			egen tmp = anymatch(hedib0?w2) , v(`1') 
			replace ever`x'_w2 = 1 if tmp == 1 
			drop tmp
				
			replace ever`x'_w2 = 0 if ever`x'_w2 == . ///
						& hedib01w2 >=0 & hedib01w2!=. 
		
			
		
		}

		
		
	else {

		** 					chronic / CVD 
		** feed forward: 	hedibw  / hediaw 
		** confirmation: 	hediad  / hediac 
		** dispute reason:  hediam  / hedian 
		** new report: 	    hedib0? / hedim0?
	
	** ff information 
		******************
		
			replace ff`x'_w2 = 0 if inwave2==1
			replace ff`x'_w2 = 0 if hediaw`1'w2<=-1 
			replace ff`x'_w2 = 1 if hediaw`1'w2==`1' 
		
		
		** dispute information 
		***********************

			replace disp`x'_w2 = -1  if hediac`1'w2<=-3 | hedian`1'w2<=-3 		/* missing responses */
			replace disp`x'_w2 =  1  if ff`x'_w2 == 0 		/* nothing ff */
			replace disp`x'_w2 =  2  if hediac`1'w2==1 		/* confirm */
			replace disp`x'_w2 =  3  if hedian`1'w2==1		/* never had it */
			replace disp`x'_w2 =  4  if hedian`1'w2==3		/* not prev but now */
			assert  disp`x'_w2 !=. if inwave2==1 			
		
		** ever diagnosed 
		******************
			
			* confirmation of previous diagnosis
			replace ever`x'_w2 = -1 if disp`x'_w2 == -1 	/* missing confimation info */
			replace ever`x'_w2 =  0 if disp`x'_w2 == 3 		/* never had it */
			replace ever`x'_w2 =  1 if disp`x'_w2 == 2 | disp`x'_w2 == 4 /* have now */
			
			* fresh reports 
			replace ever`x'_w2 = -1 if ever`x'_w2 == . & hedim01w2 <=-8 
			replace ever`x'_w2 =  0 if ever`x'_w2 == . & hedim01w2 == 96 
			
			egen tmp = anymatch(hedim0?w2) , v(`1') 
			replace ever`x'_w2 = 1 if tmp == 1 
			drop tmp 
			
			replace ever`x'_w2 = 0 if ever`x'_w2 == . & hedim01w2 >=0 & hedim01w2!=. 

		}
		mac shift
	}
	
	
	** check that onset information is being fed into the prevalence info. 
		assert evermi_w2 == 1 if henmmiw2>=1 & henmmiw2<=3
		
	** for some reason people are given a separate opportunity to report a 
	** heart attack diagnosis...
		replace evermi_w2 = 1 if heyrbw2==1  
		
	** one person reports that they are taking medication for 
	** their lung condition but doesn't report onset either in wave 1 or wave 2. 
	** coding as a 1 would include a new onset. Don't do this - assume that 
	** the lung condition is not a chronic one.
	
	** ta helngw2 everlu_w2 
	

	
	
	
	*-------------* 
	*- WAVES 3/7 -*
	*-------------* 	
	
		** 					chronic / CVD 
		** feed forward: 	hedbw   / hedaw 
		** confirmation: 	hedbd   / hedac 
		** dispute reason:  hedbm   / hedan 
		** new report: 	    hedib   / hedim
		
	** rename to make looping easier 		
		
		foreach x in w c n { 
			forv i = 3/4 { 		
				gen heda`x'blw`i' = .
			}
		}
		
		ren dhedim* hedim* 	
		ren dhedib* hedib* 
		ren hedawar?? hedawhr?? 
		ren hedacar?? hedachr?? 
		ren hedanar?? hedanhr?? 
		ren hedimar?? hedimhr?? 
		
		ren hedbw???? hedaw???? 
		ren hedbd???? hedac???? 
		ren hedbm???? hedan???? 
		
		ren hedib96?? hedim95?? 
		
		forv i = 3/7 { 
			ren hedib??w`i' hedim??w`i'
		}
		
			
		
	foreach x in lu as ar os ca pd ps ad de bl bp an mi hf hm hr di st ch { 
			forv i = 3/7 { 
			
		/* 1. make sure ff info is internally consistent */
		
		** I don't care if the question or schedule is n/a
			recode hedaw`x'w`i' (-2 = -1) 
			recode hedac`x'w`i' (-2 = -1) 
			recode hedan`x'w`i' (-2 = -1) 		
		
		di in red "no further information if `x' not ff" 
			assert (hedac`x'w`i' == -1 & hedan`x'w`i' == -1) 	///
					if hedaw`x'w`i'==-1
			
		di in red "positive information later if `x' ff"
			assert hedac`x'w`i' != -1 if hedaw`x'w`i'>0 & hedaw`x'w`i'!=. 
		
		
		/* 2. make sure confirmations and disputes map on to each other */
		
		** if doesn't confirm, but then doesn't have reason for dispute, 
		** code as missing. 
			replace hedan`x'w`i' = -8 if hedac`x'w`i' == 2 & hedan`x'w`i' == -1 
		
		
		** if says 'don't have it anymore, code as confirmation ** 
		** and code the 'don't have it anymore' reason as missing **
			replace hedac`x'w`i' = 1 if hedan`x'w`i' == 2 
			recode hedan`x'w`i' (2 = -1) 
		
		** make sure only non-confirmations are 
		**	passed on for 'reason for dispute' ** 
			
			count if hedan`x'w`i' != -1 & hedac`x'w`i' == 1 
			if `r(N)' >=1 { 
				di in red "`x' wave `i'" 
				list idauniq if hedan`x'w`i' != -1 & hedac`x'w`i' == 1 
			} 			
		
			replace hedan`x'w`i' = -1 if hedac`x'w`i' == 1 
			assert (hedan`x'w`i' <=-8 | hedan`x'w`i'>=1) & hedan`x'w`i'!=. ///
					if hedac`x'w`i'==2 
			
		** sort out 'not read out as didn't make sense' responses 
		** to reflect that actually the information wasn't fed forward 
			assert hedan`x'w`i' == -1 if hedac`x'w`i' ==3 
			replace hedaw`x'w`i' = -1 if hedac`x'w`i' ==3 
			recode hedac`x'w`i' (3 = -1) 
			
		}
		}

		
	foreach x in lu as ar os ca pd ps ad de bl bp an mi hf hm hr di st ch { 
			forv i = 3/7 { 
		
		/** condition fed forward? **/ 
		
			replace ff`x'_w`i' = (hedaw`x'w`i' >-1 & hedaw`x'w`i'!=. ) if inwave`i' == 1 
			
		/** confirmation/dispute information **/	
			
			replace disp`x'_w`i' = -1 if hedac`x'w`i' <=-8 | hedan`x'w`i' <=-8 
			replace disp`x'_w`i' =  1 if ff`x'_w`i' == 0    /* not ff */
			replace disp`x'_w`i' =  2 if hedac`x'w`i' == 1  /* confirm */
			replace disp`x'_w`i' =  3 if hedan`x'w`i' == 1  /* never had it */
			replace disp`x'_w`i' =  4 if hedan`x'w`i' == 3  /* not prev but now */
			replace disp`x'_w`i' =  5 if hedan`x'w`i' == 4  /* misdiagnosed */
			
			
		/** ever diagnosed **/	
			
			replace ever`x'_w`i' = -1 if disp`x'_w`i' == -1   /* missing confimation info */
			replace ever`x'_w`i' =  0 if disp`x'_w`i' == 3 | disp`x'_w`i' == 5 	/* never had it or misdiagnosed */
			replace ever`x'_w`i' =  1 if disp`x'_w`i' == 2 | disp`x'_w`i' == 4 /* have now */
			
			* fresh reports 
			
			replace ever`x'_w`i' = -1 if ever`x'_w`i' == . & hedim`x'w`i' <=-1 
			replace ever`x'_w`i' =  hedim`x'w`i' if ever`x'_w`i' == . ///
											& hedim`x'w`i' >= 0
								
			}
		}
		
		* check consisitency with record of number of heart attacks recorded *
		forv i = 3/7 { 
			assert evermi_w`i' == 1 if henmmiw`i'>=1 & henmmiw`i'<=3
		}	
		
	 save "$healthsave\raw_prevalence_w7.dta", replace 
